{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of 558LSTM.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "S6Lxe1naQVwy",
        "colab_type": "code",
        "outputId": "7bc035f7-02b4-40ba-e681-e10ebd2962d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "!pip install fuzzywuzzy"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: fuzzywuzzy in /usr/local/lib/python3.6/dist-packages (0.17.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NZJeBFZbK3u5",
        "colab_type": "code",
        "outputId": "353ab60b-772b-4ca2-8bf3-51e65c6b99c9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import gensim\n",
        "from fuzzywuzzy import fuzz\n",
        "from nltk.corpus import stopwords\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "from scipy.stats import skew, kurtosis\n",
        "from scipy.spatial.distance import cosine, cityblock, jaccard, canberra, euclidean, minkowski, braycurtis\n",
        "from scipy.stats import skew, kurtosis\n",
        "import gensim\n",
        "from gensim.models import Word2Vec\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "from nltk import word_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "stop_words = stopwords.words('english')\n",
        "import scipy\n",
        "from tqdm import tqdm_notebook\n",
        "%matplotlib inline"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_axj15lEMEjy",
        "colab_type": "code",
        "outputId": "5e3e236f-72e3-4655-ddeb-147a63bedf4b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        }
      },
      "source": [
        "! rm -r 558-project/\n",
        "!git clone https://github.com/TYTYTYTYTYTYTYTYTY/558-project.git\n",
        "!wget -c \"https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz\""
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into '558-project'...\n",
            "remote: Enumerating objects: 47, done.\u001b[K\n",
            "remote: Counting objects:   2% (1/47)\u001b[K\rremote: Counting objects:   4% (2/47)\u001b[K\rremote: Counting objects:   6% (3/47)\u001b[K\rremote: Counting objects:   8% (4/47)\u001b[K\rremote: Counting objects:  10% (5/47)\u001b[K\rremote: Counting objects:  12% (6/47)\u001b[K\rremote: Counting objects:  14% (7/47)\u001b[K\rremote: Counting objects:  17% (8/47)\u001b[K\rremote: Counting objects:  19% (9/47)\u001b[K\rremote: Counting objects:  21% (10/47)\u001b[K\rremote: Counting objects:  23% (11/47)\u001b[K\rremote: Counting objects:  25% (12/47)\u001b[K\rremote: Counting objects:  27% (13/47)\u001b[K\rremote: Counting objects:  29% (14/47)\u001b[K\rremote: Counting objects:  31% (15/47)\u001b[K\rremote: Counting objects:  34% (16/47)\u001b[K\rremote: Counting objects:  36% (17/47)\u001b[K\rremote: Counting objects:  38% (18/47)\u001b[K\rremote: Counting objects:  40% (19/47)\u001b[K\rremote: Counting objects:  42% (20/47)\u001b[K\rremote: Counting objects:  44% (21/47)\u001b[K\rremote: Counting objects:  46% (22/47)\u001b[K\rremote: Counting objects:  48% (23/47)\u001b[K\rremote: Counting objects:  51% (24/47)\u001b[K\rremote: Counting objects:  53% (25/47)\u001b[K\rremote: Counting objects:  55% (26/47)\u001b[K\rremote: Counting objects:  57% (27/47)\u001b[K\rremote: Counting objects:  59% (28/47)\u001b[K\rremote: Counting objects:  61% (29/47)\u001b[K\rremote: Counting objects:  63% (30/47)\u001b[K\rremote: Counting objects:  65% (31/47)\u001b[K\rremote: Counting objects:  68% (32/47)\u001b[K\rremote: Counting objects:  70% (33/47)\u001b[K\rremote: Counting objects:  72% (34/47)\u001b[K\rremote: Counting objects:  74% (35/47)\u001b[K\rremote: Counting objects:  76% (36/47)\u001b[K\rremote: Counting objects:  78% (37/47)\u001b[K\rremote: Counting objects:  80% (38/47)\u001b[K\rremote: Counting objects:  82% (39/47)\u001b[K\rremote: Counting objects:  85% (40/47)\u001b[K\rremote: Counting objects:  87% (41/47)\u001b[K\rremote: Counting objects:  89% (42/47)\u001b[K\rremote: Counting objects:  91% (43/47)\u001b[K\rremote: Counting objects:  93% (44/47)\u001b[K\rremote: Counting objects:  95% (45/47)\u001b[K\rremote: Counting objects:  97% (46/47)\u001b[K\rremote: Counting objects: 100% (47/47)\u001b[K\rremote: Counting objects: 100% (47/47), done.\u001b[K\n",
            "remote: Compressing objects: 100% (35/35), done.\u001b[K\n",
            "remote: Total 47 (delta 16), reused 43 (delta 12), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (47/47), done.\n",
            "--2019-11-07 02:05:27--  https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz\n",
            "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.216.108.109\n",
            "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.216.108.109|:443... connected.\n",
            "HTTP request sent, awaiting response... 206 Partial Content\n",
            "Length: 1647046227 (1.5G), 1367029602 (1.3G) remaining [application/x-gzip]\n",
            "Saving to: ‘GoogleNews-vectors-negative300.bin.gz’\n",
            "\n",
            "GoogleNews-vectors- 100%[+++================>]   1.53G  62.9MB/s    in 23s     \n",
            "\n",
            "2019-11-07 02:05:50 (57.4 MB/s) - ‘GoogleNews-vectors-negative300.bin.gz’ saved [1647046227/1647046227]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mHbApf9FLQAR",
        "colab_type": "code",
        "outputId": "a83759f0-7dfd-4358-f386-573f0ff6b423",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 161
        }
      },
      "source": [
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from pandas import DataFrame\n",
        "import numpy as np \n",
        "train = pd.read_csv(\"558-project/data/train.csv\")\n",
        "test = pd.read_csv(\"558-project/data/test.csv\")\n",
        "\n",
        "#train = train.drop(\"id\",1, inplace=True)\n",
        "#test = test.drop(\"id\",1, inplace=True)\n",
        "print(train.head(5))\n"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "   id  ... same_source\n",
            "0   1  ...           1\n",
            "1   2  ...           0\n",
            "2   3  ...           0\n",
            "3   4  ...           0\n",
            "4   5  ...           0\n",
            "\n",
            "[5 rows x 4 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kmPChmBZLiAT",
        "colab_type": "code",
        "outputId": "d850c756-d0f8-4b78-d978-a23307ea4866",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "model = gensim.models.KeyedVectors.load_word2vec_format( \\\n",
        "    'GoogleNews-vectors-negative300.bin.gz', binary=True)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z6wxgKycPdmT",
        "colab_type": "code",
        "outputId": "945db5ed-0b97-48c8-ddec-6b8c779be68e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "def sent2vec(s,model):\n",
        "    words = str(s).lower()\n",
        "    words = word_tokenize(words)\n",
        "    words = [w for w in words if not w in stop_words]\n",
        "    words = [w for w in words if w.isalpha()]\n",
        "    M = []\n",
        "    for w in words:\n",
        "        try:\n",
        "            M.append(model[w])\n",
        "        except:\n",
        "            continue\n",
        "    M = np.array(M)\n",
        "    v = M.sum(axis=0)\n",
        "    return v / np.sqrt((v ** 2).sum())\n",
        "\n",
        "#sent1 vector\n",
        "train_sent1_vectors = np.zeros((train.shape[0], 300))\n",
        "for i, s in enumerate(tqdm_notebook(train.sent1.values)):\n",
        "    train_sent1_vectors[i, :] = sent2vec(s,model)\n",
        "    \n",
        "#sent2 vector    \n",
        "train_sent2_vectors  = np.zeros((train.shape[0], 300))\n",
        "for i, s in enumerate(tqdm_notebook(train.sent2.values)):\n",
        "    train_sent2_vectors[i, :] = sent2vec(s,model) "
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2cfc36bb70224f3fbe286926aa1db063",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=129156), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:14: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7cfb248ea38b4b2ebc6537f3c5c69c46",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=129156), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b7CZAVeYPlkA",
        "colab_type": "code",
        "outputId": "b3a1af38-74dc-40f5-c736-96bc4014fe6b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "#sent1 vector\n",
        "test_sent1_vectors = np.zeros((test.shape[0], 300))\n",
        "for i, s in enumerate(tqdm_notebook(test.sent1.values)):\n",
        "    test_sent1_vectors[i, :] = sent2vec(s,model)\n",
        "    \n",
        "#sent2 vector    \n",
        "test_sent2_vectors  = np.zeros((test.shape[0], 300))\n",
        "for i, s in enumerate(tqdm_notebook(test.sent2.values)):\n",
        "    test_sent2_vectors[i, :] = sent2vec(s,model)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dce77ea04d4e436dadf54e2955477e01",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=14350), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:14: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "63931c1d72084de78e26a59b4c9634b7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=14350), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qBEJ7EzaPqlS",
        "colab_type": "code",
        "outputId": "b940076d-d4eb-4902-b96e-572605525250",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "train['cosine_distance'] = [cosine(x, y) for (x, y) in zip(np.nan_to_num(train_sent1_vectors), np.nan_to_num(train_sent2_vectors))]\n",
        "train['cityblock_distance'] = [cityblock(x, y) for (x, y) in zip(np.nan_to_num(train_sent1_vectors), np.nan_to_num(train_sent2_vectors))]\n",
        "train['jaccard_distance'] = [jaccard(x, y) for (x, y) in zip(np.nan_to_num(train_sent1_vectors), np.nan_to_num(train_sent2_vectors))]\n",
        "train['canberra_distance'] = [canberra(x, y) for (x, y) in zip(np.nan_to_num(train_sent1_vectors), np.nan_to_num(train_sent2_vectors))]\n",
        "train['euclidean_distance'] = [euclidean(x, y) for (x, y) in zip(np.nan_to_num(train_sent1_vectors), np.nan_to_num(train_sent2_vectors))]\n",
        "train['minkowski_distance'] = [minkowski(x, y, 3) for (x, y) in zip(np.nan_to_num(train_sent1_vectors), np.nan_to_num(train_sent2_vectors))]\n",
        "train['braycurtis_distance'] = [braycurtis(x, y) for (x, y) in zip(np.nan_to_num(train_sent1_vectors), np.nan_to_num(train_sent2_vectors))]\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "test['cosine_distance'] = [cosine(x, y) for (x, y) in zip(np.nan_to_num(test_sent1_vectors), np.nan_to_num(test_sent2_vectors))]\n",
        "test['cityblock_distance'] = [cityblock(x, y) for (x, y) in zip(np.nan_to_num(test_sent1_vectors), np.nan_to_num(test_sent2_vectors))]\n",
        "test['jaccard_distance'] = [jaccard(x, y) for (x, y) in zip(np.nan_to_num(test_sent1_vectors), np.nan_to_num(test_sent2_vectors))]\n",
        "test['canberra_distance'] = [canberra(x, y) for (x, y) in zip(np.nan_to_num(test_sent1_vectors), np.nan_to_num(test_sent2_vectors))]\n",
        "test['euclidean_distance'] = [euclidean(x, y) for (x, y) in zip(np.nan_to_num(test_sent1_vectors), np.nan_to_num(test_sent2_vectors))]\n",
        "test['minkowski_distance'] = [minkowski(x, y, 3) for (x, y) in zip(np.nan_to_num(test_sent1_vectors), np.nan_to_num(test_sent2_vectors))]\n",
        "test['braycurtis_distance'] = [braycurtis(x, y) for (x, y) in zip(np.nan_to_num(test_sent1_vectors), np.nan_to_num(test_sent2_vectors))]"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/scipy/spatial/distance.py:720: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  dist = 1.0 - uv / np.sqrt(uu * vv)\n",
            "/usr/local/lib/python3.6/dist-packages/scipy/spatial/distance.py:1178: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  return l1_diff.sum() / l1_sum.sum()\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "64DoMpX0eU81",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train['skew_s1vec'] = [skew(x) for x in np.nan_to_num(train_sent1_vectors)]\n",
        "train['skew_s2vec'] = [skew(x) for x in np.nan_to_num(train_sent2_vectors)]\n",
        "train['kur_s1vec'] = [kurtosis(x) for x in np.nan_to_num(train_sent1_vectors)]\n",
        "train['kur_s2vec'] = [kurtosis(x) for x in np.nan_to_num(train_sent2_vectors)]\n",
        "\n",
        "\n",
        "test['skew_s1vec'] = [skew(x) for x in np.nan_to_num(test_sent1_vectors)]\n",
        "test['skew_s2vec'] = [skew(x) for x in np.nan_to_num(test_sent2_vectors)]\n",
        "test['kur_s1vec'] = [kurtosis(x) for x in np.nan_to_num(test_sent1_vectors)]\n",
        "test['kur_s2vec'] = [kurtosis(x) for x in np.nan_to_num(test_sent2_vectors)]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qmKUfOiYPrWx",
        "colab_type": "code",
        "outputId": "ed861194-74c7-48f3-fa01-924b6a4f8166",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "train_ckpt_distance = train[['cosine_distance','cityblock_distance','jaccard_distance',\n",
        "                'canberra_distance','euclidean_distance','minkowski_distance','braycurtis_distance',\n",
        "                 'skew_s1vec','skew_s2vec','kur_s1vec','kur_s2vec']]\n",
        "print(train_ckpt_distance.shape)\n",
        "\n",
        "\n",
        "\n",
        "test_ckpt_distance = test[['cosine_distance','cityblock_distance','jaccard_distance',\n",
        "                'canberra_distance','euclidean_distance','minkowski_distance','braycurtis_distance',\n",
        "                 'skew_s1vec','skew_s2vec','kur_s1vec','kur_s2vec']]\n",
        "print(test_ckpt_distance.shape)\n",
        "\n",
        "del(model)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(129156, 11)\n",
            "(14350, 11)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SZ4ozj61PuVR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def wmd(s1, s2):\n",
        "    s1 = str(s1).lower().split()\n",
        "    s2 = str(s2).lower().split()\n",
        "    stop_words = stopwords.words('english')\n",
        "    s1 = [w for w in s1 if w not in stop_words]\n",
        "    s2 = [w for w in s2 if w not in stop_words]\n",
        "    return model.wmdistance(s1, s2)\n",
        "\n",
        "def norm_wmd(s1, s2):\n",
        "    s1 = str(s1).lower().split()\n",
        "    s2 = str(s2).lower().split()\n",
        "    stop_words = stopwords.words('english')\n",
        "    s1 = [w for w in s1 if w not in stop_words]\n",
        "    s2 = [w for w in s2 if w not in stop_words]\n",
        "    return norm_model.wmdistance(s1, s2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KYu509yCP1bx",
        "colab_type": "code",
        "outputId": "839a5ca8-ab25-4fa8-b123-eed30980b39a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "model = gensim.models.KeyedVectors.load_word2vec_format( \\\n",
        "    'GoogleNews-vectors-negative300.bin.gz', binary=True)\n",
        "\n",
        "train['wmd'] = train.apply(lambda x: wmd(x['sent1'], x['sent2']), axis=1)\n",
        "\n",
        "test['wmd'] = test.apply(lambda x: wmd(x['sent1'], x['sent2']), axis=1)\n",
        "\n",
        "del(model)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XtjFgw-6P4K9",
        "colab_type": "code",
        "outputId": "c52c4453-4750-4f29-ccda-fc995c9e0e88",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "norm_model = gensim.models.KeyedVectors.load_word2vec_format( \\\n",
        "    'GoogleNews-vectors-negative300.bin.gz', binary=True)\n",
        "norm_model.init_sims(replace=True)\n",
        "\n",
        "train['norm_wmd'] = train.apply(lambda x: norm_wmd(x['sent1'], x['sent2']), axis=1)\n",
        "\n",
        "test['norm_wmd'] = test.apply(lambda x: norm_wmd(x['sent1'], x['sent2']), axis=1)\n",
        "\n",
        "del(norm_model)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-7_x91K9P9lZ",
        "colab_type": "code",
        "outputId": "c4579044-10c0-446c-9067-ded609b617b0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        }
      },
      "source": [
        "train_ckpt_wmd=train[['wmd','norm_wmd']]\n",
        "train_ckpt_wmd\n",
        "\n",
        "test_ckpt_wmd=test[['wmd','norm_wmd']]\n",
        "test_ckpt_wmd"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>wmd</th>\n",
              "      <th>norm_wmd</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3.234672</td>\n",
              "      <td>1.289266</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3.363726</td>\n",
              "      <td>1.239698</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4.202454</td>\n",
              "      <td>1.360028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4.247422</td>\n",
              "      <td>1.341462</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3.347305</td>\n",
              "      <td>1.280948</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14345</th>\n",
              "      <td>3.681080</td>\n",
              "      <td>1.334166</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14346</th>\n",
              "      <td>3.670352</td>\n",
              "      <td>1.341901</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14347</th>\n",
              "      <td>3.405346</td>\n",
              "      <td>1.288707</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14348</th>\n",
              "      <td>3.574445</td>\n",
              "      <td>1.313100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14349</th>\n",
              "      <td>3.142913</td>\n",
              "      <td>1.159968</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>14350 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            wmd  norm_wmd\n",
              "0      3.234672  1.289266\n",
              "1      3.363726  1.239698\n",
              "2      4.202454  1.360028\n",
              "3      4.247422  1.341462\n",
              "4      3.347305  1.280948\n",
              "...         ...       ...\n",
              "14345  3.681080  1.334166\n",
              "14346  3.670352  1.341901\n",
              "14347  3.405346  1.288707\n",
              "14348  3.574445  1.313100\n",
              "14349  3.142913  1.159968\n",
              "\n",
              "[14350 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2iW1rc-4kkl-",
        "colab_type": "text"
      },
      "source": [
        "## embedding matrix \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FfTTi-f5kj3b",
        "colab_type": "code",
        "outputId": "a4ebf1f9-c129-429e-e684-c5af13a7d110",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 82
        }
      },
      "source": [
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FJy_MvAskvpI",
        "colab_type": "code",
        "outputId": "900b6c62-322f-4c3c-9cd7-1caf3c313b50",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "texts=pd.concat([train.sent1, train.sent2], ignore_index= True)\n",
        "\n",
        "print(texts.shape)\n",
        "\n",
        "NUM_WORDS=20000\n",
        "tokenizer = Tokenizer(num_words=NUM_WORDS,filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n\\'',\n",
        "                      lower=True)\n",
        "tokenizer.fit_on_texts(texts)\n",
        "sequences_sent1_train = tokenizer.texts_to_sequences(train.sent1)\n",
        "sequences_sent2_train = tokenizer.texts_to_sequences(train.sent2)\n",
        "sequences_sent1_test = tokenizer.texts_to_sequences(test.sent1)\n",
        "sequences_sent2_test = tokenizer.texts_to_sequences(test.sent2)\n",
        "sequences_train = tokenizer.texts_to_sequences(texts)\n",
        "word_index = tokenizer.word_index\n",
        "print('Found %s unique tokens.' % len(word_index))\n",
        "\n"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(258312,)\n",
            "Found 140450 unique tokens.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LNGFXMCLU4tE",
        "colab_type": "code",
        "outputId": "72eda414-7c06-420d-9d50-194aeda0a629",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 305
        }
      },
      "source": [
        "sequences_train = pad_sequences(sequences_train)\n",
        "pad_sent1_train,pad_sent2_train = train_test_split(sequences_train,test_size=0.5, shuffle= False )\n",
        "pad_sent1_test = pad_sequences(sequences_sent1_test,maxlen= sequences_train.shape[1])\n",
        "pad_sent2_test = pad_sequences(sequences_sent2_test,maxlen= sequences_train.shape[1])\n",
        "\n",
        "print(pad_sent1_train.shape)\n",
        "print(pad_sent2_train.shape)\n",
        "print(pad_sent1_test.shape)\n",
        "print(pad_sent2_test.shape)\n",
        "\n",
        "print(pad_sent1_train[8,:])\n"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(129156, 167)\n",
            "(129156, 167)\n",
            "(14350, 167)\n",
            "(14350, 167)\n",
            "[   0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    1   28   76    7 4101   21   52   95   99  847   53   93]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OeC9IYkzncuk",
        "colab_type": "code",
        "outputId": "7b7921ba-b8d8-402d-cd5e-ac5285bf21ee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "import gensim\n",
        "from gensim.models import Word2Vec\n",
        "from gensim.utils import simple_preprocess\n",
        "\n",
        "from gensim.models.keyedvectors import KeyedVectors\n",
        "\n",
        "word_vectors = KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin.gz', binary=True)\n",
        "\n",
        "EMBEDDING_DIM=300\n",
        "vocabulary_size=len(word_index)+1\n",
        "embedding_matrix = np.zeros((vocabulary_size, EMBEDDING_DIM))\n",
        "for word, i in word_index.items():\n",
        "    if i>=NUM_WORDS:\n",
        "        continue\n",
        "    try:\n",
        "        embedding_vector = word_vectors[word]\n",
        "        embedding_matrix[i] = embedding_vector\n",
        "    except KeyError:\n",
        "        embedding_matrix[i]=np.random.normal(0,np.sqrt(0.25),EMBEDDING_DIM)\n",
        "\n",
        "del(word_vectors)\n",
        "\n"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JODXTfncSSUJ",
        "colab_type": "code",
        "outputId": "135c1c39-9f84-4d0c-d29d-b70a49bc60f8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print(embedding_matrix.shape)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(140451, 300)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iBLANRYAYjJA",
        "colab_type": "code",
        "outputId": "5fcf5e5b-fa7b-496c-e9e1-b9bea2e9908c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "mm_train = np.hstack((train_ckpt_wmd,train_ckpt_distance))\n",
        "mm_train.shape\n",
        "\n",
        "mm_test = np.hstack((test_ckpt_wmd,test_ckpt_distance))\n",
        "mm_test.shape\n",
        "\n"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(14350, 13)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q6iM2xsLaUXM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "np.random.seed(0)\n",
        "\n",
        "from keras.models import Model\n",
        "from keras.layers import Dense, Input, Dropout, LSTM, Activation, BatchNormalization,concatenate,Subtract, Dot, Multiply,Bidirectional,Lambda\n",
        "from keras.layers.embeddings import Embedding\n",
        "from keras.initializers import glorot_uniform\n",
        "from keras.layers.noise import GaussianNoise\n",
        "from keras import backend as K\n",
        "from keras import optimizers\n",
        "import tensorflow as tf\n",
        "\n",
        "import keras.callbacks as kcallbacks\n",
        "np.random.seed(1)\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4pUuiT0YaZ3u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_model():\n",
        "    embedding_layer = Embedding(vocabulary_size,\n",
        "                                EMBEDDING_DIM,\n",
        "                                weights=[embedding_matrix],\n",
        "                                input_length=167,\n",
        "                                trainable=False)\n",
        "\n",
        "    sent_1 = Input(shape=(167,), dtype='int32')\n",
        "    y1 = embedding_layer(sent_1)\n",
        "    \n",
        "\n",
        "    sent_2 = Input(shape=(167,), dtype='int32')\n",
        "    y2 = embedding_layer(sent_2)\n",
        "   \n",
        "    \n",
        "    shared_lstm_1 = LSTM(75, return_sequences=True)\n",
        "    shared_lstm_2 = LSTM(75)\n",
        "\n",
        "    q1 = shared_lstm_1(y1)\n",
        "    q1 = Dropout(0.5)(q1)\n",
        "    q1 = BatchNormalization()(q1)\n",
        "    q1 = shared_lstm_2(q1)\n",
        "        # q1 = Dropout(0.5)(q1)\n",
        "\n",
        "    q2 = shared_lstm_1(y2)\n",
        "    q2 = Dropout(0.5)(q2)\n",
        "    q2 = BatchNormalization()(q2)\n",
        "    q2 = shared_lstm_2(q2)\n",
        "\n",
        "    d = Subtract()([q1, q2])\n",
        "    \n",
        "    distance = Multiply()([d, d])\n",
        "\n",
        "    angle = Multiply()([q1, q2])\n",
        "\n",
        "    magic_input = Input(shape=(13,))\n",
        "    magic_dense = BatchNormalization()(magic_input)\n",
        "    magic_dense = Dense(64, activation='relu')(magic_dense)\n",
        "        #magic_dense = Dropout(0.3)(magic_dense)\n",
        "        \n",
        "    merged = concatenate([distance,angle,magic_dense])\n",
        "    merged = Dropout(0.1)(merged)\n",
        "    merged = BatchNormalization()(merged)\n",
        "\n",
        "    merged = Dense(256, activation='relu')(merged)  # 64\n",
        "    merged = Dropout(0.1)(merged)\n",
        "    merged = BatchNormalization()(merged)\n",
        "\n",
        "    merged = Dense(64, activation='relu')(merged)  # 64\n",
        "    merged = Dropout(0.1)(merged)\n",
        "    merged = BatchNormalization()(merged)\n",
        "\n",
        "    is_duplicate = Dense(1, activation='sigmoid')(merged)\n",
        "\n",
        "    model = Model(inputs=[sent_1, sent_2, magic_input], outputs=is_duplicate)\n",
        "\n",
        "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "\n",
        "    model.summary()\n",
        "    return model\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D_SbaWe3bHiY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "1eb3e1fd-871a-4faa-ae5d-4e4ab0dcc365"
      },
      "source": [
        "model = get_model()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:190: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:197: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:203: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:207: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:216: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:223: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:148: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3733: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:793: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3657: The name tf.log is deprecated. Please use tf.math.log instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/nn_impl.py:183: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            (None, 167)          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_2 (InputLayer)            (None, 167)          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)         (None, 167, 300)     42135300    input_1[0][0]                    \n",
            "                                                                 input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lstm_1 (LSTM)                   (None, 167, 75)      112800      embedding_1[0][0]                \n",
            "                                                                 embedding_1[1][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dropout_1 (Dropout)             (None, 167, 75)      0           lstm_1[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "dropout_2 (Dropout)             (None, 167, 75)      0           lstm_1[1][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_1 (BatchNor (None, 167, 75)      300         dropout_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_2 (BatchNor (None, 167, 75)      300         dropout_2[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "lstm_2 (LSTM)                   (None, 75)           45300       batch_normalization_1[0][0]      \n",
            "                                                                 batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "input_3 (InputLayer)            (None, 13)           0                                            \n",
            "__________________________________________________________________________________________________\n",
            "subtract_1 (Subtract)           (None, 75)           0           lstm_2[0][0]                     \n",
            "                                                                 lstm_2[1][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_3 (BatchNor (None, 13)           52          input_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "multiply_1 (Multiply)           (None, 75)           0           subtract_1[0][0]                 \n",
            "                                                                 subtract_1[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "multiply_2 (Multiply)           (None, 75)           0           lstm_2[0][0]                     \n",
            "                                                                 lstm_2[1][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 64)           896         batch_normalization_3[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 214)          0           multiply_1[0][0]                 \n",
            "                                                                 multiply_2[0][0]                 \n",
            "                                                                 dense_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dropout_3 (Dropout)             (None, 214)          0           concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_4 (BatchNor (None, 214)          856         dropout_3[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 256)          55040       batch_normalization_4[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "dropout_4 (Dropout)             (None, 256)          0           dense_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_5 (BatchNor (None, 256)          1024        dropout_4[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 64)           16448       batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "dropout_5 (Dropout)             (None, 64)           0           dense_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_6 (BatchNor (None, 64)           256         dropout_5[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dense_4 (Dense)                 (None, 1)            65          batch_normalization_6[0][0]      \n",
            "==================================================================================================\n",
            "Total params: 42,368,637\n",
            "Trainable params: 231,943\n",
            "Non-trainable params: 42,136,694\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kGIYi0p_aqFF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mm_s_train, mm_val = train_test_split(mm_train, test_size = 0.01, random_state = 12357)\n",
        "pad_s_sent1_train, pad_sent1_val = train_test_split(pad_sent1_train, test_size = 0.01, random_state = 12357)\n",
        "pad_s_sent2_train, pad_sent2_val = train_test_split(pad_sent2_train, test_size = 0.01, random_state = 12357)\n",
        "y_s_train, y_val = train_test_split(train.same_source, test_size = 0.01, random_state = 12357)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jztBMxtSagD-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 485
        },
        "outputId": "b622eacc-53fe-4a5e-d47c-0dd3a621e97a"
      },
      "source": [
        "early_stopping = kcallbacks.EarlyStopping(monitor='val_loss', patience=3)\n",
        "bst_model_path = 'best_model.h5'\n",
        "model_checkpoint = kcallbacks.ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=True)\n",
        "\n",
        "hist = model.fit([pad_s_sent1_train, pad_s_sent2_train,mm_s_train], y_s_train, \\\n",
        "                 validation_data=([pad_sent1_val, pad_sent2_val, mm_val], y_val), \\\n",
        "                 epochs=50, batch_size=256, shuffle=True, callbacks=[early_stopping, model_checkpoint])\n",
        "predicts = model.predict([pad_sent1_test, pad_sent2_test,mm_test], batch_size=10, verbose=1)\n"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 127864 samples, validate on 1292 samples\n",
            "Epoch 1/50\n",
            "127864/127864 [==============================] - 578s 5ms/step - loss: 0.5585 - acc: 0.7079 - val_loss: 0.5389 - val_acc: 0.7229\n",
            "Epoch 2/50\n",
            "127864/127864 [==============================] - 580s 5ms/step - loss: 0.5381 - acc: 0.7244 - val_loss: 0.5152 - val_acc: 0.7322\n",
            "Epoch 3/50\n",
            "127864/127864 [==============================] - 579s 5ms/step - loss: 0.5228 - acc: 0.7361 - val_loss: 0.5000 - val_acc: 0.7500\n",
            "Epoch 4/50\n",
            "127864/127864 [==============================] - 581s 5ms/step - loss: 0.5100 - acc: 0.7438 - val_loss: 0.5066 - val_acc: 0.7415\n",
            "Epoch 5/50\n",
            "127864/127864 [==============================] - 580s 5ms/step - loss: 0.4969 - acc: 0.7535 - val_loss: 0.4893 - val_acc: 0.7531\n",
            "Epoch 6/50\n",
            "127864/127864 [==============================] - 569s 4ms/step - loss: 0.4861 - acc: 0.7610 - val_loss: 0.4874 - val_acc: 0.7585\n",
            "Epoch 7/50\n",
            "127864/127864 [==============================] - 566s 4ms/step - loss: 0.4768 - acc: 0.7669 - val_loss: 0.4944 - val_acc: 0.7539\n",
            "Epoch 8/50\n",
            "127864/127864 [==============================] - 577s 5ms/step - loss: 0.4673 - acc: 0.7727 - val_loss: 0.4862 - val_acc: 0.7593\n",
            "Epoch 9/50\n",
            "127864/127864 [==============================] - 577s 5ms/step - loss: 0.4599 - acc: 0.7787 - val_loss: 0.4743 - val_acc: 0.7647\n",
            "Epoch 10/50\n",
            "127864/127864 [==============================] - 576s 5ms/step - loss: 0.4519 - acc: 0.7831 - val_loss: 0.4841 - val_acc: 0.7632\n",
            "Epoch 11/50\n",
            "127864/127864 [==============================] - 570s 4ms/step - loss: 0.4426 - acc: 0.7881 - val_loss: 0.4836 - val_acc: 0.7562\n",
            "Epoch 12/50\n",
            "127864/127864 [==============================] - 569s 4ms/step - loss: 0.4348 - acc: 0.7928 - val_loss: 0.4754 - val_acc: 0.7663\n",
            "14350/14350 [==============================] - 636s 44ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fwYvlU4BbM6z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        },
        "outputId": "d19d0612-c336-4e40-d1a2-e69413c93e93"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pX1dqklrbjLE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "outputId": "ea1b8aef-3029-4675-e016-cd54efefee3d"
      },
      "source": [
        "!ls /content/gdrive/My\\ Drive/"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " 558project\n",
            "'Art history 6a.docx'\n",
            "'Colab Notebooks'\n",
            " ess9.gslides\n",
            "'Gardner&#39%3s+Art+through+the+Ages+The+Weste....pdf'\n",
            "'Geog W12 Lab 7 Website'\n",
            "'Glob notes'\n",
            "'https:  support.google.com installer answer su.png'\n",
            " IMG_4724.JPG\n",
            " pastmidterm_sol.gdoc\n",
            "'“Places I’ve Visited in California.gtable'\n",
            " projects\n",
            " Untitled0.ipynb\n",
            "'Welcome back, Kimi! | Scuderia Ferrari (1).png'\n",
            "'Welcome back, Kimi! | Scuderia Ferrari (2).png'\n",
            "'Welcome back, Kimi! | Scuderia Ferrari.png'\n",
            "'作死大旅游 MK II.gslides'\n",
            " 圣诞作死大旅游.gslides\n",
            " 未命名的表单.gform\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1lTOhwp9M2IK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "outputId": "c0adcb32-5f81-44c5-a0d0-153dacdc5283"
      },
      "source": [
        "print(predicts)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.02444449]\n",
            " [0.9795735 ]\n",
            " [0.02872616]\n",
            " ...\n",
            " [0.9616339 ]\n",
            " [0.08862007]\n",
            " [0.879396  ]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ZcXmP_Oba_p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "root_path = '/content/gdrive/My Drive/'\n",
        "\n",
        "out= pd.DataFrame()\n",
        "out['id'] = test['id']\n",
        "out['same_source'] = predicts\n",
        "\n",
        "out.to_csv(root_path+'output.csv', index = False)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}